package com.g2room.driving.grap.article.qg;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrabSgzxDataByProxy {
	static String imgPath = "sg";
	public String baseUrl = "http://img.szhk.com";
	private String soruce = "深港在线";
	private int count = 0;
	static List<String> titles = GenericArti.getTitles();
	
	public static void main(String[] args) throws Exception{
		GrabSgzxDataByProxy gkd = new GrabSgzxDataByProxy();
//		gkd.getNav();
//		
		gkd.grapImages();
    }
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		Map<String, String> pages = new HashMap<String, String>();
		// 深圳夜生活、一夜情小说、一夜情故事
		pages.put("http://www.szhk.com/news/list/gb/282673175453586/18.htm", "夜来香");
		// 明星绯闻、明星八卦绯闻、韩国明星情侣-深港在线名人情事栏目
		pages.put("http://www.szhk.com/news/list/gb/282673175453596/17.htm", "名人情事");
		// 爱情故事、情人日记、情爱故事
		pages.put("http://www.szhk.com/news/list/gb/282673175453584/15.htm", "一片深情");
		
		
		// 男欢女爱小说、男欢女爱电影、男欢女爱视频
		pages.put("http://www.szhk.com/news/list/gb/282673175453599/39.htm", "男欢女爱");
		
		// 校园恋爱故事、恋爱宝典、恋爱技巧仅在深圳恋爱网  -
		pages.put("http://www.szhk.com/news/list/gb/282673175453592/62.htm", " 恋爱物语");
		// 情感倾诉、真情告白 -
		pages.put("http://www.szhk.com/news/list/gb/282673175453588/61.htm", "倾述空间");
		// 婚姻故事、婚前婚后、美丽闪婚 -
		pages.put("http://www.szhk.com/news/list/gb/282673175453594/63.htm", "婚姻家庭");
		
		// 剩女嫁豪门、剩女单身日记、剩男剩女
		pages.put("http://www.szhk.com/news/list/gb/282673175453590/29.htm", "单身贵族");
 
        for(String key : pages.keySet()) {
	        String pageTotal = key.substring(key.lastIndexOf("/") + 1, key.lastIndexOf("."));
	        for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
//	        	//http://www.szhk.com/news/list/gb/282673175453594/21.htm
//	        	if(i<38) continue;
	        	
	        	String url = key.replaceAll(pageTotal + ".htm", i+".htm");
	        	boolean rs = true;
	        	rs = getPage(url, pages.get(key));
	        	if(!rs) break;
	        }
        }
        System.out.println("本次抓取数据：" + count);
		
		// 最后输出失败的链接到数据库
		GenericArti.insertError2Db(soruce);
	}
	
	public void grapImages() {
      List<Integer> ids = GenericArti.getLatestIds(GenericArti.getPublishTimeStr(1), soruce);
		// 抓取图片
		if(null != ids && ids.size() > 0) {
			HandleImages.getContentImgs(ids, baseUrl, imgPath);
//			// 抓取缩略图
//			HandleImages.getThumbImgs(ids, baseUrl, imgPath);
		}
	}
	
	/**
	 * @param url
	 * @param typeName
	 * @return
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName + "====" + url);
		Document doc = GenericGrap.getDocByIp(url);
		Elements els = doc.getElementsByAttributeValue("class","sznews_list").get(0).getElementsByTag("li");
		List<ArticlesTo> artis = new ArrayList<ArticlesTo>();
		for(Element el : els) {
			try {
				// 获取标题
				Element titleEl = el.getElementsByTag("a").get(0);
				String titleTime = titleEl.text(); // 含了时间
				int idx = titleTime.lastIndexOf("200");
				if(idx == -1) {
					idx = titleTime.lastIndexOf("201");
				}
				String title = titleTime.substring(0, idx).trim();
				
				if(titles.contains(title)) {
					// 后续改为
					//return false;
					continue;
				}
				String time = titleTime.substring(idx);
				ArticlesTo arti = new ArticlesTo();
				arti.setTitle(title);
				arti.setSource(soruce);
				arti.setSourceUrl(titleEl.attr("href"));
				arti.setTypeName(typeName);
				arti.setPublishTime(time);
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(ArticlesTo arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}

	public void getDetail(ArticlesTo arti) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		String url = arti.getSourceUrl();
		try {
			Document doc = GenericGrap.getDocByIp(url);
			String content = getContent(doc, 0);
			// 获取分页信息
			Elements els = doc.getElementsByAttributeValue("class", "endPageNum");
			if(null != els && els.size()>0) {
				els =els.get(0).getElementsByTag("a");
				if(null != els && els.size() > 0) { 
					// 获取最后一页数据
					Element el = els.get(els.size() - 2);
					String href = el.attr("href");
					String pageTotal = el.text();
					if(pageTotal.equals("下一页")) return;
					
					for(int i = 1; i<Integer.valueOf(pageTotal); i++) {
						// 799470_3.html
						String pageUrl = url.substring(0, url.lastIndexOf("/")+1) + href.substring(0, href.lastIndexOf("_")+1) + (i+1) + ".html";
						Document pageDoc = GenericGrap.getDocByIp(pageUrl);
						content += "\n"+getContent(pageDoc, i);;
			        }
				}
			}
			content = GenericArti.handleSpace(HtmlUtil.delAllTag(content));
			arti.setContent("<div class=\"article\">" + content + "</div>");
			GenericArti.insert2Db(arti);
//			GenericGrap.titles.add(arti.getTitle());
			count++;
        } catch (Exception e) {
        	System.out.println(url);
        	GenericArti.errorUrls.put(url, arti.getTypeName());
        	e.printStackTrace();
        	return;
        }
	}
	
	public static String getContent(Document doc, int page) {
		Element el = doc.getElementById("articlecontent");
		if(null == el) {
			el = doc.getElementsByAttributeValue("class", "bd_Article").get(0);
		}
		return el.html();
	}
}
